/*
 * HandleClientThread.java
 *
 * Created on June 18, 2002, 7:03 PM
 */

package com.xtera.nuwave.tools.redundancy.redman;

import java.net.*;
import java.io.*;
import java.util.*;
import java.lang.*;
import java.text.*;
import org.apache.log4j.*;

/**
 *
 * @author  llaranje
 * @version 
 */
public class HandleClientThread extends Thread {

    /** Creates new HandleClientThread */
        
        Socket clso;
        String connType = null;
        String state = null;
        String procname = null;
        CompMon compmon = null;
        CompMon.SubCompMon subcompmon = null;
        AppMonitor appmon = null;
        BufferedReader in = null;
        PrintWriter out = null;
        boolean newconnection = false;
        boolean redman = false;
        boolean backup = false;
        boolean redundant = false;
        String disthost = null;
        int appindex = -1;
        
        private static Category trace = Category.getInstance(HandleClientThread.class);
        
        private boolean findApplication(String appname) {
            int i;
            boolean retval = false;

            // Find the application monitor object for this process
            for (i = 0; i < RedMan.appnum; i++) {
                if (RedMan.appmons[i].name.equals(appname)) {
                    boolean wait = false;
                    trace.debug("findApplication() - Will acquire RedMan.restartsem semaphore");
                    synchronized (RedMan.restartsem) {
                        trace.debug("findApplication() - Acquired RedMan.restartsem semaphore");
                        wait = RedMan.appmons[i].restartinprogress;
                        trace.debug(appname + ":restartinprogress = " + wait);                    
                        if (wait) {                       
                            trace.debug("Will wait for application to be restarted");
                            try {
                                RedMan.restartsem.wait();
                                trace.debug("Got out of wait for application to be restarted");
                            }
                            catch (Exception e) {
                                trace.warn("Wait() interrupted - Exception: " + e.getMessage());
                            }                           
                        }
                    }
                  
/**************
                    while (doloop) {
                        trace.debug(appname + " being restarted. Waiting 2 secs");
                        try {
                            Thread.sleep(2000);
                        }
                        catch (InterruptedException e) {
                            trace.debug(appname + ": sleep interrupted - Exception = " + e.getMessage());
                        }
                        trace.debug("Will try to acquire RedMan.restartsem semaphore (2) and read restartinprogress");
                        synchronized (RedMan.restartsem) {
                            doloop = RedMan.appmons[i].restartinprogress;
                            trace.debug(appname + ":restartinprogress = " + doloop + " (2)");
                        }
                        if (!doloop) {
                            trace.debug(appname + " already restarted - Proceeding ...");
                        }
                    }
 ***************/
                    
                    appmon = RedMan.appmons[i];
                    appindex = i;
                    retval = true;
                    break;
                }
            }
            
            return(retval);
        }
        
        private boolean findApplicationProcess(String name) {
            int i, j;
            
            for (i = 0; i < appmon.compnum; i++) {
                if (name.equals(appmon.compmons[i].name)) {
                    compmon = appmon.compmons[i];
                    return(true);
                }
                else if (appmon.compmons[i].type.equals("PROCESS_GROUP")) {
                    for (j = 0; j < appmon.compmons[i].subcompnum; j++) {
                        if (name.equals(appmon.compmons[i].subcompmons[j].pname)) {
                            compmon = appmon.compmons[i];
                            subcompmon = appmon.compmons[i].subcompmons[j];
                            return(true);
                        }
                    }
                }
           }
            return(false);
        }
        
        private boolean checkApplication(String msg) {
            StringTokenizer st = new StringTokenizer(msg);
            String type = st.nextToken();               // This must be "RAPP" or "OPINT"    
            if (type.equals("RAPP")) st.nextToken();    // If "RAPP", this must be "CMD" or "STATE"
            String appname = st.nextToken();
            
            trace.debug("msg = " + msg + ", appname = " + appname);
            if (appname.equals("REDMAN")) {
                String cmd = st.nextToken();
                if (!cmd.equals("SHUTDOWN") && !cmd.equals("SHUTDOWN-F") 
                                            && !cmd.equals("GETPID")) return(false);
                redman = true;
                return(true);
            }
     
                          
            return(findApplication(appname));
        }
        
        private boolean isCommCheck(String msg) {
            trace.debug("Checking if it is a handshake message...");
            StringTokenizer st = new StringTokenizer(msg);
            String type = st.nextToken(); // This must be "RAPP" to get here
            String token = st.nextToken();
            if (!token.equals("HANDSHAKE")) return(false);   // This must be "HANDSHAKE"
            String appname = st.nextToken();
            if (!appname.equals(appmon.name)) return(false); // This must be <appname>
            String host = st.nextToken();
            if (appmon.isInList(host, appmon.remotehosts)) {
                trace.debug("This is a HANDSHAKE message!");
                return(true);  // This must be a remote host
            }
            else return(false);
        }
        
        private boolean isRemStateChange(String msg) {
            StringTokenizer st = new StringTokenizer(msg);
            String type = st.nextToken(); // This must be "RAPP" to get here
            String token = st.nextToken();
            if (!token.equals("STATE")) return(false);   // This must be "STATE"
            else return(true);
        }
        
        private boolean isRemCommand(String msg) {
            StringTokenizer st = new StringTokenizer(msg);
            String type = st.nextToken(); // This must be "RAPP" to get here
            String token = st.nextToken();
            if (!token.equals("CMD")) return(false);   // This must be "CMD"
            else return(true);
        }

        private boolean checkRegistration(String msg) {
            int i;
            
            StringTokenizer st = new StringTokenizer(msg);
            String type = st.nextToken();      // This must be "APP" to get here    
            String appname = st.nextToken();
                          
            if (findApplication(appname) == false) return(false);

            String compname = st.nextToken();
                        
            // Check if the process belongs to this application
            if (findApplicationProcess(compname)) {
                String registration = st.nextToken();
                if (registration.equals("REGISTRATION")) return(true);
            }
            return(false);
        }
        
        private boolean checkDistRegistration(String msg) {
            int i;
            
            StringTokenizer st = new StringTokenizer(msg);
            String type = st.nextToken();      // This must be "RAPP" to get here 
            String appname = st.nextToken();
                          
            if (!findApplication(appname)) return(false);
            
            disthost = st.nextToken();
            
            if (!appmon.checkDistHost(disthost, out)) return(false);
                        
            String registration = st.nextToken();
            if (registration.equals("REGISTRATION")) {
                return(true);
            }
            else return(false);
        }
       
        private void processRemStateChange(String msg) {
            trace.debug(appmon.name + ": in processRemStateChange() msg = \"" + msg + "\"");
            String state, remhost, remstate, appname, newstate;
            StringTokenizer st = new StringTokenizer(msg);
            st.nextToken();  // This must be "RAPP"
            st.nextToken();  // This must be "STATE"
            appname = st.nextToken();  // This must be the application name
            if (!appname.equals(appmon.name)) return;
            remhost = st.nextToken();  // This must be the remote host name
            if (!appmon.isInList(remhost, appmon.remotehosts)) return;            
            remstate = st.nextToken();  // This is the new remote state    
            state = appmon.getSMAState();
            trace.info(appmon.name + ": processing remote state change \"" + msg 
                                   + "\", local state = " + state + ", switchinprogress = " 
                                   + appmon.switchinprogress);
            
            if (remstate.equals("STANDBY") && state.equals("STANDBY") && appmon.switchinprogress) {
                // Do not submit remote event STANDBY since a switch is in progress.
                // Set the remote state to STANDBY directly.                
                appmon.makeLocalActive();
                appmon.setSMARemoteState("STANDBY");
                newstate = appmon.getSMAState();
                if (!state.equals(newstate))
                    trace.debug("New state due to SWITCH command = " + newstate);
                appmon.switchinprogress = false;
            }   
            else {
                String cause = null;
                if (remstate.equals("FAILED")) {
                    if (st.hasMoreTokens()) {
                        cause = st.nextToken();
                        if (cause.equals("PRIMARY")) cause = "FLT";
                    }
                    else cause = "FLT";
                    remstate += " " + cause;
                }
                newstate = appmon.submitEventToSMA("RAPP " + remstate);
                if (!state.equals(newstate))
                    trace.debug("New state due to remote state change (" + remstate + ") = " + newstate);                
                       
                // If the remote state is OFFLINE it means that the remote app instance was shutdown
                // (through a RESTART or SHUTDOWN command). 
                // In case of a RESTART, the remote app object will be garbage collected and the connection 
                // with this RedMan will be broken. Therefore, we need to set the outbound connection to null 
                // as well. This is only through, though, if the remote state was something different and
                // is now becoming OFFLINE.
                if (remstate.equals("OFFLINE")) {
                    boolean remrestart = false;
                    if (st.hasMoreTokens()) {
                        String shutdownreason = st.nextToken();
                        if (shutdownreason.equals("RESTART")) remrestart = true;
                    }
                        
                    String curremstate = appmon.getSMARemoteState();
                    if (!curremstate.equals("UNKNOWN") && remrestart) appmon.outToRemoteRedMan = null;
                }
            }

            // appmon.notifyAppOfRemStateChange(remhost + " " + remstate);
            appmon.notifyAppOfRemStateChange(remstate);
        }
        
        private void processRemCommand(String msg) {                        
            String remhost, cmd, appname, state, remstate, newstate, newprimaryhost = null;
            StringTokenizer st = new StringTokenizer(msg);
            st.nextToken();  // This must be "RAPP"
            st.nextToken();  // This must be "CMD"
            appname = st.nextToken();  // This must be the application name
            if (!appname.equals(appmon.name)) return;
            remhost = st.nextToken();  // This must be the remote host name
            if (!appmon.isInList(remhost, appmon.remotehosts)) return;            
            cmd = st.nextToken();  // This is the command
            if (cmd.equals("PRIMARY")) newprimaryhost = st.nextToken();
              
            
            trace.info(appmon.name + ": processing remote command \"" + msg + "\"");
            state = appmon.getSMAState();
            if (cmd.equals("SWITCH")) {
                remstate = appmon.getSMARemoteState();
                if (state.equals("STANDBY") && remstate.equals("ACTIVE")) {
                    appmon.submitEventToSMA("APP ACTIVE");
                }
                else if (state.equals("ACTIVE")) appmon.submitEventToSMA("APP STANDBY");
            }
            else if (cmd.equals("INHIBIT")) appmon.inhibitFailoverLocal();
            else if (cmd.equals("UNINHIBIT")) appmon.uninhibitFailoverLocal();
            else if (cmd.equals("PRIMARY")) appmon.setNewPrimaryHost(newprimaryhost, false);
            newstate = appmon.getSMAState();
            if (!state.equals(newstate))
                trace.debug("Due to remote cmd " + cmd + " app " 
                                                 + appmon.name + " changed states from " 
                                                 + state + " to " + newstate);
        }
        
        private void processAcknowledgment(String msg) {
            try {
                if (msg.equals("ACK")) {
                    trace.info("Processing ACK message from " + appmon.name + ":" + compmon.name);                    

                    synchronized (appmon.stateChgTask) {
                        trace.debug("Will check stateChgTask");
                        if (appmon.stateChgTask != null) {   
                            trace.debug("stateChgTask not null!");
                            appmon.countmsgs++;
                            trace.debug("countmsgs = " + String.valueOf(appmon.countmsgs)
                                                              + ", nummanaged = "
                                                              + String.valueOf(appmon.nummanaged));                            
                            if (appmon.countmsgs == appmon.nummanaged) {
                                trace.debug("countmsgs matches nummanaged");
                                appmon.countmsgs = 0;
                                if (appmon.local) {
                                    trace.debug("App is local; will commit new state -----------------------------------------------");
                                    String state = appmon.commitNewSMAState();
                                    trace.debug("New state committed ---------------------------------------------------------------");
                                    String debmsg = "All managed processes ";
                                    if (appmon.numdistmngdmsgs != 0) debmsg += "and remote components ";                                    
                                    debmsg += "(" + String.valueOf(appmon.nummanaged) 
                                                           + ") of application " 
                                                           + appmon.name + " changed state to " 
                                                           + state;
                                    trace.debug(debmsg);
                                }
                                else {
                                    appmon.stateChgTask.cancel();
                                    appmon.stateChgTask = null;
                                    String debmsg = "All managed processes "; 
                                    if (appmon.numdistmngdmsgs != 0) debmsg += "and remote components ";
                                    debmsg += "(" + String.valueOf(appmon.nummanaged)
                                                           + ") of application " 
                                                           + appmon.name 
                                                           + " on this host changed state"; 
                                    trace.debug(debmsg);
                                    appmon.notifyApplicationMonitor("ACK");
                                }
                            }
                            else {
                                trace.debug("Received ACK msg, but still waiting for ACKs");
                            }
                        }
                        else {
                            trace.debug("Received ACK msg when stateChgThread is null");
                        }
                    }
                }
                else if (msg.equals("NACK")) {
                    synchronized (appmon.stateChgTask) {
                        appmon.countmsgs = 0;
                        appmon.stateChgTask.cancel();
                        appmon.stateChgTask = null;
                        if (appmon.local) appmon.handleFailure("FLT");
                        else appmon.handleDistFailure("FLT");
                    }
                }
            }
            catch (Exception e) {  // SocketException and IOException
                trace.error("Exception: " + e.getMessage());
            }
        }
        
        private void processDistAcknowledgment(String msg) {
            try {
                if (msg.equals("ACK")) {
                    synchronized (appmon.stateChgTask) {
                        if (appmon.stateChgTask != null) {                       
                            appmon.countmsgs++;
                            trace.debug("countmsgs = " + String.valueOf(appmon.countmsgs)
                                                              + ", nummanaged = "
                                                              + String.valueOf(appmon.nummanaged));
                            if (appmon.countmsgs == appmon.nummanaged) {
                                appmon.countmsgs = 0;
                                String state = appmon.commitNewSMAState();
                                String debmsg = "All managed processes ";
                                if (appmon.numdistmngdmsgs != 0) debmsg += "and remote components ";                               
                                debmsg += "(" + String.valueOf(appmon.nummanaged) 
                                                       + ") of application " 
                                                       + appmon.name + " changed state to " 
                                                       + state; 
                                trace.debug(debmsg);
                            }
                        }
                    }
                }
                else if (msg.equals("NACK")) {
                    synchronized (appmon.stateChgTask) {
                        appmon.countmsgs = 0;
                        appmon.stateChgTask.cancel();
                        appmon.stateChgTask = null;
                        appmon.handleFailure("FLT");
                    }
                }
            }
            catch (Exception e) {  // SocketException and IOException
                trace.error("Exception: " + e.getMessage());
            }
        }
        
        private boolean processInitStateRequest(String msg) {            
            StringTokenizer st = new StringTokenizer(msg);
            //st.nextToken();  // This must be "APP"
            //String appname = st.nextToken();  // This must be the application name
            //if (!appname.equals(appmon.name)) return(false);
            //String procname = st.nextToken();  // This must be the process name
            String event = st.nextToken();  // This is the event
            
            trace.debug("Processing init message: " + msg + ", event = " + event 
                                            + ", initializing = " + appmon.initializing);
            
            if (event.equals("REQ_INIT_STATE") && 
                 (((subcompmon == null) && compmon.managed) || 
                  ((subcompmon != null) && subcompmon.submanaged))) {
                if (appmon.initializing) {
                    appmon.countmsgs++;
                    trace.debug("Number of init msgs received = " 
                                            + String.valueOf(appmon.countmsgs));
                    trace.debug("Number of managed processes = " 
                                            + String.valueOf(appmon.nummanaged));
                    if (appmon.countmsgs == appmon.nummanaged) {
                        appmon.initializing = false;
                        if (appmon.local) {                           
                            String debmsg = "All managed processes ";
                            if (appmon.numdistmngdmsgs != 0) debmsg += "and remote components ";
                            debmsg += "(" + appmon.nummanaged
                                                   + ") of application " 
                                                   + appmon.name + " are initialized";
                            trace.debug(debmsg);
                            appmon.countmsgs = 0;
                            String newstate = appmon.submitEventToSMA("APP INITIALIZED");
                            trace.debug("Newstate due to INITIALIZED event = " + newstate); 
                        }
                        else {
                            trace.debug("All managed processes of application " 
                                                    + appmon.name 
                                                    + " on this host are initialized");
                            appmon.countmsgs = 0;
                            appmon.notifyApplicationMonitor("INITIALIZED");
                        }
                    }
                    return(true);
                }
                else {
                    String targetstate = appmon.getSMATargetState();
                    TimerTask spawner = null;
                    if (subcompmon != null) {
                        subcompmon.notifySubCompOfStateChange(targetstate);
                         
                        // Create task that will wait to spawn subcomponent test threads
                        spawner = new TimerTask() {
                             public void run() { 
                                trace.debug("Thread that will wait to spawn test threads for"
                                                          + " restarted subcomponent " 
                                                          + subcompmon.pname + " is running ...");
                                subcompmon.waitToSpawnSubCompTestThreads(); 
                             }
                        };
                    }
                    else {
                        compmon.notifyLocalCompOfStateChange(targetstate);
                        
                        // Create task that will wait to spawn component test threads
                        spawner = new TimerTask() {
                             public void run() {
                                trace.debug("Thread that will wait to spawn test threads for"
                                                           + " restarted uniprocess component " 
                                                           + compmon.name + " is running ...");
                                 compmon.waitToSpawnUniprocessCompTestThreads(); 
                             }
                        };                       
                    }
                    
                    // Schedule thread that will wait to spawn the component/subcomponent test threads
                    Timer timer = new Timer();
                    timer.schedule(spawner, 1);
                    
                    return(true);
                }
            }
            else return(false);
        }
        
        private boolean processDistInitStateRequest(String msg) {            
            StringTokenizer st = new StringTokenizer(msg);
            //st.nextToken();  // This must be "PRXY"
            //String appname = st.nextToken();  // This must be the application name
            //if (!appname.equals(appmon.name)) return(false);
            //String compname = st.nextToken();  // This must be the component name
            String event = st.nextToken();  // This is the event
            
            trace.debug("Processing init message: " + msg);
            
            if (event.equals("REQ_INIT_STATE")) {
                if (appmon.initializing) {
                    appmon.countmsgs++;
                    trace.debug("Number of init msgs received = " 
                                            + String.valueOf(appmon.countmsgs));
                    System.out.print("Number of managed processes ");
                    if (appmon.numdistmngdmsgs != 0) {
                        System.out.print("and remote components = " 
                                            + String.valueOf(appmon.nummanaged));
                        if (appmon.countmsgs == appmon.nummanaged) {
                            appmon.initializing = false;
                            String debmsg = "All managed processes ";
                            if (appmon.numdistmngdmsgs != 0) debmsg += "and remote components ";
                            debmsg += "(" + String.valueOf(appmon.nummanaged)
                                                   + ") of application " + appmon.name 
                                                   + " are initialized";
                            appmon.countmsgs = 0;
                            String newstate = appmon.submitEventToSMA("APP INITIALIZED");
                            trace.debug("Newstate due to INITIALIZED event = " + newstate);           

                            return(true);
                        }
                    }
                }
                return(true);
            }
            else return(false);
        }
        
        private boolean processErrorNotification(String msg) {
            trace.debug("processErrorNotification() - Error msg = '" + msg + "'");
            // Verify message header
            StringTokenizer st = new StringTokenizer(msg);
            //st.nextToken();  // This must be "APP"
            //String appname = st.nextToken();  // This must be the application name
            //if (!appname.equals(appmon.name)) return;
            //String procname = st.nextToken();  // This must be the process name
            //if (!compmon.name.equals(procname)) {
                //if ((subcompmon != null) && (!subcompmon.pname.equals(procname))) return;
            //}
            
            if (!(st.nextToken()).equals("ERROR")) {
                trace.debug(appmon.name + ": connection thread received unrecognized message '" + msg + "'");
                return(true);
            }
            
            if (!st.hasMoreTokens()) {
                trace.debug(appmon.name + ": connection thread received unrecognized error message '" + msg + "'");
                return(true);
            }
                
            String errorType = st.nextToken();
            
            if (errorType.equals("FAULT")) {
                if (subcompmon != null) CompMon.killPid(subcompmon.subcomppid);
                else CompMon.killPid(compmon.comppid);
                return(false);
            }
            
            if (errorType.equals("FAILURE")) {
                String cause = "FLT";
                trace.debug("cause = '" + cause + "'");
                if (st.hasMoreTokens()) cause = st.nextToken();
                trace.debug("cause = '" + cause + "'");
                if (appmon.stateChgTask != null) {
                    synchronized (appmon.stateChgTask) {
                        appmon.stateChgTask.cancel();
                        appmon.stateChgTask = null;
                    }
                }
                appmon.countmsgs = 0;
                if (appmon.local) {
                    if (cause.equals("DB")) appmon.handleFailure("DB");
                    else appmon.handleFailure("FLT");
                }
                else {
                    if (cause.equals("DB")) appmon.handleDistFailure("DB");
                    else appmon.handleDistFailure("FLT");
                }
                return(false);
            }
            
            // Error is a warning
            return(true);
        }
        
        private void processTestResponse(String msg) {
            //trace.debug("Test response received: " + msg);
            StringTokenizer st = new StringTokenizer(msg);            
            if (!(st.nextToken()).equals("TESTACK")) return;
            String tname = "PING";
            
            // Find the corresponding TestThread
            if (subcompmon == null) {
                for (int i = 0; i < compmon.numtests; i++) {
                    if (compmon.tests[i].testname.equals(tname)) {
                        compmon.tests[i].sendTestResponse(msg);
                        break;
                    }
                }
            }
            else {
                for (int i = 0; i < subcompmon.numtests; i++) {
                    if (subcompmon.tests[i].testname.equals(tname)) {
                        subcompmon.tests[i].sendTestResponse(msg);
                        break;
                    }
                }                
            }
        }
        
        private void processDistFailureNotification(String msg) {
            // Verify message header
            StringTokenizer st = new StringTokenizer(msg);
            st.nextToken();  // This must be "PRXY"
            String appname = st.nextToken();  // This must be the application name
            if (!appname.equals(appmon.name)) return;
            
            String remhost = st.nextToken();  // This must be the name of the remote host
            
            String event = st.nextToken();  // This must be "FAILURE"
            if (!(st.nextToken()).equals("FAILURE")) return;
            String cause = st.nextToken();          
            synchronized (appmon.stateChgTask) {
                if (appmon.stateChgTask != null) {
                    appmon.stateChgTask.cancel();
                    appmon.stateChgTask = null;
                }
                appmon.countmsgs = 0;
                if (appmon.local) {
                    if (cause.equals("DB")) appmon.handleFailure("DB");
                    else appmon.handleFailure("FLT");
                }
                else {
                    if (cause.equals("DB")) appmon.handleDistFailure("DB");
                    else appmon.handleDistFailure("FLT");
                }
            }
        }
        
        public HandleClientThread (Socket so) {
            clso = so;
            
            try {
                // Get streams to talk to client
                in  = new BufferedReader(new InputStreamReader(clso.getInputStream()));
                out = new PrintWriter(new OutputStreamWriter(clso.getOutputStream()));
            }
            catch (Exception e) {
                trace.error("Exception: " + e.getMessage());
            }
            
            newconnection = true;
        }
        
        private void sendBasicInformation() {
            String info = "ACK ";
            if (appmon.hamodel.equals("1")) info += "SIMPLEX";
            else {
                info += "REDUNDANT " + appmon.remotehost;
                if (appmon.designation) info += " PRIMARY";
                else info += " SECONDARY";
                if (appmon.failoverenabled) info += " UNINHIBITED";
                else info += " INHIBITED";
            }
            out.println(info);
            out.flush();
        }
              
        private void handleAPPConnection(String regmsg) {
            boolean init = true;
            try {
                if (checkRegistration(regmsg)) {                       
                    if (compmon.type.equals("PROCESS_GROUP")) {
                        trace.debug(appmon.name + ":" + compmon.name + ":" 
                                                + subcompmon.pname + " process registration ok");
                        subcompmon.out = out;
                    }
                    else {
                        trace.debug(appmon.name + ":" + compmon.name + " process registration ok");                        
                        compmon.out = out;
                    }                
                    sendBasicInformation();
                }
                else {
                    trace.warn(appmon.name + ": msg \"" + regmsg + "\" did not pass registration test.");
                    return;
                }
                
                for (String msg = null; (msg = in.readLine()) != null;) {
                    if (subcompmon != null) {
                        trace.debug(appmon.name + ":" + compmon.name + ":" + subcompmon.pname
                                           + ": conn thread received msg \"" 
                                           + msg + "\", length = " + msg.length());
                    }
                    else {
                        trace.debug(appmon.name + ":" + compmon.name
                                           + ": conn thread received msg \"" 
                                           + msg + "\", length = " + msg.length());                        
                    }
                    if (msg.length() == 0) break;
                    msg = msg.trim();

                    if (msg.equals("REQ_INIT_STATE")) {
                        if (subcompmon != null) {
                            trace.debug("APP connection thread: " + appmon.name + ":" 
                                            + compmon.name + ":" + subcompmon.pname
                                            + " still initializing");
                        }
                        else {
                            trace.debug("APP connection thread: " + appmon.name + ":" 
                                            + compmon.name + " still initializing");                            
                        }
                        if (processInitStateRequest(msg)) init = false;                        
                    }
                    else if (msg.equals("ACK") || (msg.equals("NACK"))) {
                        if (appmon.stateChgTask == null) {
                            if (subcompmon != null) {
                                trace.debug("APP connection thread: " + appmon.name + ":"
                                                    + compmon.name + ":" + subcompmon.pname
                                                    + " received msg \"" 
                                                    + msg + "\" when not changing state");
                            }
                            else {
                                trace.debug("APP connection thread: " + appmon.name + ":"
                                                    + compmon.name + " received msg \"" 
                                                    + msg + "\" when not changing state");
                            }
                            continue;
                        }
                        processAcknowledgment(msg);
                    }
                    else if (msg.startsWith("ERROR")) {
                        boolean willsurvive = processErrorNotification(msg);
                        if (!willsurvive) {
                            trace.debug(appmon.name + ": APP connection thread exiting because process failed due to SW or DB fault");
                            return;
                        }
                    }
                    else processTestResponse(msg);
                }
                if (subcompmon != null) {
                    trace.warn(appmon.name + ":" + compmon.name + ":" + subcompmon.pname 
                                       + " connection thread got empty msg or broken connection.");
                }
                else {
                    trace.warn(appmon.name + ":" + compmon.name
                                       + " connection thread got empty msg or broken connection");                    
                }
                return;               
            }
            catch (Exception e) {
                trace.error("APP connection thread got exception: '" + e.getMessage() + "' - Will exit");
            }
        }
        
        private void handlePRXYConnection(String regmsg) {
            boolean init = true;
            try {
                if (checkDistRegistration(regmsg)) {
                    trace.debug("PRXY connection thread: remote registration ok.");
                }
                else {
                    trace.warn(appmon.name + ": conn thread msg \"" 
                                           + regmsg + "\" did not pass registration test");
                    return;
                }
                
                for (String msg = null; (msg = in.readLine()) != null;) {
                    trace.info(appmon.name + ": conn thread received msg \"" 
                                           + msg + "\", length = " + msg.length());                      
                    if (msg.length() == 0) break;
                    msg = msg.trim();
                    if (init) {
                        if (processDistInitStateRequest(msg)) init = false;
                    }
                    else if (msg.equals("ACK") || (msg.equals("NACK"))) {
                        if (appmon.stateChgTask == null) {
                            trace.warn(appmon.name + ": conn thread received msg \""
                                                   + msg + "\", when not changing state");
                            continue;
                        }
                        processDistAcknowledgment(msg);
                    }
                    else processDistFailureNotification(msg);
                }
                trace.warn(appmon.name + ":" + compmon.name + ":" + subcompmon.pname 
                                       + " - conn thread got empty msg or broken connection.");
                return;               
            }
            catch (Exception e) {
                trace.error("Exception: " + e.getMessage());
            }
        }
        
        private void execStartCmd() {
            trace.debug(appmon.name + ": OPINT connection thread executing START command");           
            if (appmon == null) {
                trace.debug("OPINT connection thread: application not found");
                out.println("NACK - Application " + appmon.name + " not found");
                out.flush();
            }
            if (appmon.running) {
                trace.debug(appmon.name + ": (OPINT connection thread) application already running on this host");
                out.println("NACK - " + appmon.name + " is already running on this host");
                out.flush();
                return;
            }
            
            appmon.bootstart = true;
            appmon.startenabled = true;
                       
            String res = appmon.handleStart("START");
            if (res.equals("OK")) {
                trace.debug("OPINT connection thread: application " + appmon.name + " started");
                out.println("ACK - Application " + appmon.name + " started");
                out.flush();
            }
            else {
                out.println("NACK - " + res);
                out.flush();
                trace.debug("OPINT connection thread: application " + appmon.name + " NOT started");
            }
        }
        
        private String shutdownOneApp(AppMonitor appmon1) {
            trace.debug(appmon1.name + ": OPINT connection thread executing SHUTDOWN command");
            if (!appmon1.running) {
                return("ACK - Application " + appmon.name 
                                                 + " had already been shutdown on this host");
            }           
            String res = appmon1.handleShutdown("SHUTDOWN");
            if (res.equals("OK")) {
                return("ACK - Application is shutdown");
            }
            else {
                return("NACK - " + res);
            }            
        }
        
        private String isAnyRedundantApplicationActive() {
            String state, hamodel;
            for (int i = 0; i < RedMan.appnum; i++) {
                if (RedMan.appmons[i].running) {
                    state = RedMan.appmons[i].getSMAState();
                    hamodel = RedMan.appmons[i].hamodel;
                    if (state.equals("ACTIVE") && !hamodel.equals("1")) 
                        return(RedMan.appmons[i].name);
                }
            }
            return(null);
        }
        
        private String isAnyApplicationInTransition() {
            String state;
            for (int i = 0; i < RedMan.appnum; i++) {
                if (RedMan.appmons[i].running) {
                    state = RedMan.appmons[i].getSMAState();
                    if (state.startsWith("IN TRANSITION"))
                        return(RedMan.appmons[i].name);
                }
            }
            return(null);
        }
    
        private void shutdownAllAppsAndRedMan (boolean force) {
            String app, apps = null, res;
            if (!force) {
                if ((app = isAnyRedundantApplicationActive()) != null) {
                    out.println("NACK - Application " + app + " is ACTIVE");
                    out.flush();
                    return;                
                }
                if ((app = isAnyApplicationInTransition()) != null) {
                    out.println("NACK - Application " + app + " is IN TRANSITION");
                    out.flush();
                    return;                                    
                }
            }
            int num = 0;
            for (int i = 0; i < RedMan.appnum; i++) {
                if (RedMan.appmons[i].running) {
                    res = shutdownOneApp(RedMan.appmons[i]);
                    if (!res.startsWith("ACK")) {
                        if (apps == null) apps = RedMan.appmons[i].name;
                        else apps += ", " + RedMan.appmons[i];
                        num++;
                    }
                }
            }
            
            if (num == 0) {
                try {
                    out.println("ACK - All applications shutdown - RedMan exiting");
                    out.flush();
                }
                catch (Exception e) {
                    trace.debug("Exception: could not send ACK confirming applications shutdown - " 
                                                                                    + e.getMessage());
                }
                                
                // Tell Backup RedMan to exit
                if (RedMan.backupwriter != null) {
                    RedMan.backupwriter.println("REDMAN SHUTDOWN");
                    RedMan.backupwriter.flush();
                    trace.debug("PRIMARY RedMan sent SHUTDOWN msg to BACKUP RedMan");
                }
                trace.info("All applications shutdown - RedMan exiting");
                System.exit(0);
            }
            else {
                res = "NACK - ";
                if (num > 1) res += "Applications (" + apps + ")";
                else res += "Application " + apps;
                res += " failed to shutdown - RedMan not exiting";
                out.println(res);
                out.flush();
            }
        }
                
        private void execShutdownCmd(boolean force) {
            if (redman) shutdownAllAppsAndRedMan(force);
            else {
                String state = appmon.getSMAState();
                if (state.equals("ACTIVE") && !appmon.hamodel.equals("1") && !force) {
                    out.println("NACK - Application is ACTIVE");
                    out.flush();
                    return;
                }
                String res = shutdownOneApp(appmon);
                out.println(res);
                out.flush();
            }
        }
                      
        private void execEnableCmd() {
            String res = null;
            trace.debug("OPINT connection thread: executing ENABLE command");
            appmon.startenabled = true;

            if (!appmon.running) {
                res = appmon.handleStart("ENABLE");
                if (res.equals("OK")) {
                    out.println("ACK - Application " + appmon.name + " start enabled and started");
                    out.flush();
                }
                else {
                    out.println("NACK - " + res);
                    out.flush();
                }
            }
            else {
                res = appmon.handleStart("CFGFILE");
                if (res.equals("OK")) {
                    out.println("ACK - Already running application " 
                                     + appmon.name + " was enabled for automatic start");
                    out.flush();
                }
                else {
                    out.println("NACK - " + res);
                    out.flush();
                }
            }
        }
        
        private void execDisableCmd(boolean force) {
            String state = appmon.getSMAState();
            if (appmon.initializing && !force) {
                out.println("NACK - " + appmon.name + " is initializing");
                out.flush();
                return;                
            }
            if (state.startsWith("IN TRANSITION") && !force) {
                out.println("NACK - " + appmon.name + " is undergoing a state transition");
                out.flush();
                return;
            }
            if (state.equals("ACTIVE") && !appmon.hamodel.equals("1") && !force) {
                out.println("NACK - " + appmon.name + " is ACTIVE");
                out.flush();
                return;
            }
            trace.debug("OPINT connection thread: executing DISABLE command");
            String res = null;
            appmon.startenabled = false;
            if (appmon.running) {
                res = appmon.handleShutdown("DISABLE");
                if (res.equals("OK")) {
                    out.println("ACK - Application shutdown and automatic start disabled");
                    out.flush();
                }
                else {
                    out.println("NACK - " + res);
                    out.flush();
                }
            }
            else {
                res = appmon.handleShutdown("CFGFILE");
                if (res.equals("OK")) {
                    out.println("ACK - Already shutdown " + appmon.name 
                                                          + " application was start disabled");
                    out.flush();
                }
                else {
                    out.println("NACK - " + res);
                    out.flush();
                }
            }
        }
            
        private void execInhibitCmd() {
            if (appmon.hamodel.equals("1")) {
                out.println("NACK - " + appmon.name + " HA model (simplex) does not support this command");
                out.flush();
                return;
            }
            String res = appmon.inhibitFailoverLocal();
            if (res.equals("OK")) {
                res = appmon.inhibitFailoverRemote();
                if (res.equals("OK")) out.println("ACK - Failover/switchover inhibited for application " 
                                                                              + appmon.name);
                else out.println("NACK - " + res);
            }
            else out.println("NACK - " + res);           
            out.flush();
        }
        
        private void execUninhibitCmd() {
            if (appmon.hamodel.equals("1")) {
                out.println("NACK - " + appmon.name + " HA model (simplex) does not support this command");
                out.flush();
                return;
            }
            //String state = appmon.getSMAState();
            //if (state.equals("MAINTENANCE")) {
            //    out.println("NACK - " + appmon.name + " in state MAINTENANCE - Use command UNTEST");
            //    out.flush();
            //    return;
            //}
            
            String res = appmon.uninhibitFailoverLocal();
            if (res.equals("OK")) {
                res = appmon.uninhibitFailoverRemote();
                if (res.equals("OK")) out.println("ACK - Failover/switchover uninhibited for application " 
                                                                                + appmon.name);
                else out.println("NACK - " + res);
            }
            else out.println("NACK - " + res);
            out.flush();
        }
        
        private void setSwitchInProgress(String remstate) {
            String newstate = null, newremstate = null, res = null;
            if (remstate.equals("ACTIVE")) {    // The remote app is ACTIVE
                appmon.switchinprogress = true;
                res = appmon.makeRemoteStandby();
                if (!res.equals("OK")) {
                    appmon.switchinprogress = false;
                    return;
                }
                int time = 0;
                for (;;) {
                    try {
                        Thread.sleep(1000);
                    }
                    catch (InterruptedException e) {
                        trace.debug("Thread interrupted in sleep() during switchover: " + e.getMessage());
                    }
                    time += 1000;
                    if (time > appmon.statechgtime) {
                        appmon.switchinprogress = false;
                        break;
                    }
                }
            }
            else if (remstate.equals("STANDBY")) {   // The remote app is STANDBY
                appmon.switchinprogress = true;
                res = appmon.makeLocalStandby();
                if (!res.equals("OK")) {
                    appmon.switchinprogress = false;
                    return;
                }
                int time = 0;
                for (;;) {
                    try {
                        Thread.sleep(1000);
                    }
                    catch (InterruptedException e) {
                        trace.debug("Thread interrupted in sleep() during switchover: " + e.getMessage());
                    }
                    time += 1000;
                    if (time > appmon.statechgtime) {
                        appmon.switchinprogress = false;
                        break;
                    }
                }               
            }
        }
              
        private void execSwitchCmd() {
            String res = null;
            if (appmon.hamodel.equals("1")) {
                out.println("NACK - " + appmon.name + " HA model (simplex) does not support this command");
                out.flush();
                return;
            }
            if (!appmon.failoverenabled) {
                out.println("NACK - Application failover/switchover is inhibited");
                out.flush();
                return;
            }
            if (appmon.switchinprogress) {
                trace.debug(appmon.name + ": NACK - Switch rejected - A switch is already in progress");
                out.println("NACK - A SWITCH is already in progress");
                out.flush();
                return;
            }
            String state = appmon.getSMAState();
            if (state.equals("ACTIVE")) {
                String remstate = appmon.getSMARemoteState();
                if (remstate.equals("STANDBY")) {
                    if (appmon.isCommunicatingWithPeer()) {
                        trace.debug(appmon.name + ": ACK switch in progress");
                        out.println("ACK - SWITCH of application is in progress");
                        out.flush();
                        setSwitchInProgress("STANDBY");
                    }
                    else {
                        trace.debug(appmon.name + ": no communication with remote application instance");
                        out.println("NACK - no communication with remote application instance");
                        out.flush();                        
                    }
                }
                else {
                    trace.debug(appmon.name + ": application pair not in ACTIVE-STANDBY state");
                    out.println("NACK - Application pair not in ACTIVE-STANDBY state");
                    out.flush();
                }
            }
            else if (state.equals("STANDBY")) {
                String remstate = appmon.getSMARemoteState();
                if (remstate.equals("ACTIVE")) {
                    if (appmon.isCommunicatingWithPeer()) {
                        trace.debug(appmon.name + ": switch in progress");
                        out.println("ACK - SWITCH of application is in progress");
                        out.flush();
                        setSwitchInProgress("ACTIVE");
                    }
                    else {
                        trace.debug(appmon.name + ": no communication with remote application instance");
                        out.println("NACK - no communication with remote application instance");
                        out.flush();                        
                    }
                }
                else {
                    trace.debug(appmon.name + ": application pair not in ACTIVE-STANDBY state");
                    out.println("NACK - Application pair not in ACTIVE-STANDBY state");
                    out.flush();
                }
            }
            else {
                trace.debug(appmon.name + ": application pair not in ACTIVE-STANDBY state");
                out.println("NACK - Application pair not in ACTIVE-STANDBY state");
                out.flush();
            }
        }
        
        private void execRestartCmd(boolean force) {
            String state = appmon.getSMAState();
            if (state.equals("ACTIVE") && !force) {
                out.println("NACK - Application is ACTIVE");
                out.flush();
                return;
            }
            if (state.startsWith("IN TRANSITION") && !force) {
                out.println("NACK - " + appmon.name + " is undergoing a state transition");
                out.flush();
                return;
            }
            
            trace.debug(appmon.name + ": OPINT connection thread executing RESTART command");
            appmon.restartinprogress = true;
            if (appmon.running) {
                String res = appmon.handleShutdown("SHUTDOWN");
                trace.debug(appmon.name + ": handleShutdown() executed - res = " + res);
                if (!res.equals("OK")) {
                    out.println("NACK - " + res);
                    out.flush();
                    return;
                }
            }
                        
            if (redmancfgfilepath == null) redmancfgfilepath = RedMan.cfgfilepath;            
            
            Properties cfgtable = loadRedManCfgFile(redmancfgfilepath);
            if (cfgtable != null) {
                String appname = appmon.name;
                trace.debug(appname + " being restarted");
                trace.debug("Will acquire RedMan.restartsem to notify other threads");
                synchronized (RedMan.restartsem) {
                    trace.debug("Semaphore acquired - Will restart application");
                    out.println("ACK - Application restarted");
                    out.flush();
                    trace.debug("new AppMonitor() will be called to restart app");
                    RedMan.appmons[appindex] = new AppMonitor(appname, cfgtable, true);                
                    appmon = RedMan.appmons[appindex];

                    trace.debug("Will notify threads");
                    RedMan.restartsem.notifyAll();
                    trace.debug("Threads notified that app is restarted");
                }
                // appmon.restartinprogress = false;
            }
            else {
                out.println("NACK - Could not load RedMan configuration file");
                out.flush();
            }
        }
        
        private Properties loadRedManCfgFile(String filepath) {
            FileInputStream cfgfile = null;
            Properties cfgtable = null;
            trace.debug("RedMan config file = " + filepath);
            try {
                cfgfile = new FileInputStream(filepath);           
            }
            catch (FileNotFoundException e) {
                trace.error("Exception: specified RedMan config file not found. " + e.getMessage());
                out.println("NACK - App shutdown but not restarted - Specified RedMan config file not found");
                out.flush();
                return (null);
            }
                        
            try {
                cfgtable = new Properties();
                cfgtable.load(cfgfile);
            }
            catch (IOException e) {
                trace.error("Unable to open/read config file " + filepath 
                                                               + ": IOException,  " + e.getMessage());
                out.println("NACK - App shutdown but not restarted - Unable to open RedMan config file");
                out.flush();
                return(null);                                                                      
            }
            return(cfgtable);            
        }
        
        private void execTestCmd() {
            String state = appmon.getSMAState();
            if (state.startsWith("IN TRANSITION")) {
                out.println("NACK - " + appmon.name + " is undergoing a state transition");
                out.flush();
                return;
            }
            String res = appmon.attemptToDriveAppToMaintenance();
            if (!res.equals("OK")) {
                out.println("NACK - " + res);
            }
            else {
                out.println("ACK - Failover inhibited. App " + appmon.name 
                                          + " changing state from " + state + " to MAINTENANCE");
            }
            out.flush();
            return;
        }
        
        private void execUntestCmd() {
            String state = appmon.getSMAState();
            if (state.startsWith("IN TRANSITION")) {
                out.println("NACK - " + appmon.name + " is undergoing a state transition");
                out.flush();
                return;
            }
            String res = appmon.attemptToReleaseAppFromMaintenance();
            if (!res.startsWith("OK")) {
                out.println("NACK - " + res);
            }
            else {
                StringTokenizer st =  new StringTokenizer(res);
                st.nextToken();
                String newstate = st.nextToken();
                if (state.equals("MAINTENANCE")) {
                    if (!newstate.equals("MAINTENANCE")) 
                        out.println("ACK - Failover uninhibited. App " + appmon.name 
                                            + " changing state from MAINTENANCE to " + newstate);
                    else out.println("NACK - Failover uninhibited. MAINTENANCE app could not change state");
                }
                else out.println("ACK - Failover uninhibited. App " + appmon.name 
                                                           + " was already in state " + state);
            }
            out.flush();
            return;
        }
        
        private void execQueryCmd() {
            String state = appmon.getSMAState();
            String remstate = null;
            String msg = appmon.name + " state = (" + appmon.localhost + ":" + state;
            if (appmon.hamodel.equals("1+1") || appmon.hamodel.equals("1+0")) {
                if (appmon.designation) msg += ":PRIMARY";
                else msg += ":SECONDARY";
                remstate = appmon.getSMARemoteState();
                msg += ", " + appmon.remotehost + ":" + remstate;
                if (appmon.designation) msg += ":SECONDARY";
                else msg += ":PRIMARY";
                msg += ", ";
                if (appmon.failoverenabled) msg += "UNINHIBITED";
                else msg += "INHIBITED";
            }
            msg += ")";
            
            out.println(msg);
            out.flush();
            trace.debug(msg);
            return;
        }
        
        private void execReloadCmd() {
            String state = appmon.getSMAState();
            if (state.startsWith("IN TRANSITION")) {
                out.println("NACK - " + appmon.name + " is undergoing a state transition");
                out.flush();
                return;
            }
            
            // Copy redundant configuration files if specified
            if (redundant) {
                String path = null, cmd = null;
                try {
                    path = appmon.homedir + "/conf/" + appmon.name + "static.conf";
                    cmd = "/bin/cp " + path + ".redundant " + path;
                    Runtime.getRuntime().exec(cmd);
                    path = RedMan.cfgfilepath;
                    cmd = "/bin/cp " + path + ".redundant " + path;
                    Runtime.getRuntime().exec(cmd);
                }
                catch (Exception e) {
                    trace.debug("Failed to copy redundant config files for configuration reloading "
                                    + "(cmd = '" + cmd + "') - " + e.getMessage()); 
                    out.println("NACK - Failed to copy redundant config files for configuration reloading");
                    out.flush();
                    return;
                }
            }
            
            String res = appmon.loadAndReadNewConfiguration();
            if (res.equals("OK")) {
                out.println("ACK - New configuration loaded for application " + appmon.name);
            }
            else out.println("NACK - " + res);
            out.flush();
        }
        
        String newprimaryhost = null;
        String redmancfgfilepath = null;
        
        private void execPrimaryCmd() {
            if (appmon.hamodel.equals("1")) {
                out.println("NACK - " + appmon.name + " HA model (simplex) does not support this command");
                out.flush();
                return;
            }
            String state = appmon.getSMAState();
            if (state.startsWith("IN TRANSITION")) {
                out.println("NACK - " + appmon.name + " is undergoing a state transition");
                out.flush();
                return;
            }
            String res = appmon.setNewPrimaryHost(newprimaryhost, false);
            if (res.equals("OK")) {
                out.println("ACK - Host " + newprimaryhost + " designated as PRIMARY for " + appmon.name);
                appmon.sendCommandToRemoteRedMan("PRIMARY " + newprimaryhost);
            }
            else out.println("NACK - " + res);
            out.flush();
        }
        
        private void execActiveCmd() {
            String state;
            if (appmon.hamodel.equals("1") || !appmon.isCommunicatingWithPeer()) {
                state = appmon.getSMAState();
                if (!appmon.hamodel.equals("1") && !state.equals("STANDBY")) {
                    out.println("NACK - Application " + appmon.name + " not in state STANDBY");
                    out.flush();
                }                    
                state = appmon.submitEventToSMA("APP ACTIVE");
                if (state.equals("ACTIVE")) {
                    out.println("ACK - Application " + appmon.name + " changing state to ACTIVE");
                    out.flush();
                }
                else {
                    out.println("NACK - Could not change application " + appmon.name + " to ACTIVE");
                    out.flush();
                }
            }
            else {
                if (!appmon.hamodel.equals("1")) {
                    out.println("NACK - Redundant " + appmon.name 
                                                    + " application monitors are communicating");
                    out.flush();
                }
            }
        }
        
        private void execStandbyCmd() {
            String state;
            if (appmon.hamodel.equals("1")) {
                out.println("NACK - " + appmon.name + " HA model (simplex) does not support this command");
                out.flush();
                return;
            }
            if (!appmon.isCommunicatingWithPeer()) {
                state = appmon.getSMAState();
                if (!state.equals("ACTIVE")) {
                    out.println("NACK - Application " + appmon.name + " not in state ACTIVE");
                    out.flush();
                }                    
                state = appmon.submitEventToSMA("APP STANDBY");
                if (state.equals("STANDBY")) {
                    out.println("ACK - Application " + appmon.name + " changing state to STANDBY");
                    out.flush();
                }
                else {
                    out.println("NACK - Could not change application " + appmon.name + " to STANDBY");
                    out.flush();
                }
            }
            else {
                out.println("NACK - Redundant " + appmon.name + " application monitors are communicating");
                out.flush();
            }
        }
        
        private void execGetPidCmd() {
            String stpid;
            if (redman) {
                if (backup) {
                    int times =0;
                    while (RedMan.mybackuppid == 0) {
                        try {
                            Thread.sleep(500);
                        }
                        catch (Exception e) {
                            trace.warn("Sleep interrupted - " +  e.getMessage());
                        }
                        times++;
                        if (times > 20) break;
                    }
                    stpid = String.valueOf(RedMan.mybackuppid);
                    trace.info("BACKUP REDMAN PID = " + stpid);
                }
                else {
                    stpid = String.valueOf(RedMan.mypid);
                    trace.info("REDMAN PID = " + stpid);
                }
            }
            else {
                stpid = String.valueOf(appmon.getProcPid(procname));
                trace.info(appmon.name + ":" + procname + " PID = " + stpid);
            }
            String resp;
            if (redman) {
                resp = "REDMAN ";
                redman = false;
                if (backup) {
                    resp += "BACKUP ";
                    backup = false;
                }
            }
            else {
                resp = appmon.name + ":" + procname + " ";
            }
            resp = resp + "PID = " + stpid;
            out.println(resp);
            out.flush();
        }
                                                          
        private void processOPINTCommand(String cmd) {
            
            if (!cmd.equals("START")      &&  !cmd.equals("SHUTDOWN")   &&
                !cmd.equals("ENABLE")     &&  !cmd.equals("DISABLE")    &&
                !cmd.equals("INHIBIT")    &&  !cmd.equals("UNINHIBIT")  &&
                !cmd.equals("SWITCH")     &&  !cmd.equals("RESTART")    &&
                !cmd.equals("TEST")       &&  !cmd.equals("UNTEST")     &&
                !cmd.equals("QUERY")      &&  !cmd.equals("RELOAD")     &&
                !cmd.equals("ACTIVE")     &&  !cmd.equals("STANDBY")    &&
                !cmd.equals("SHUTDOWN-F") &&  !cmd.equals("DISABLE-F")  &&
                !cmd.equals("RESTART-F")  &&  !cmd.equals("PRIMARY")    &&
                !cmd.equals("GETPID"))                                      {
                    trace.warn("Invalid command: '" + cmd + "'");
                    out.println("NACK Invalid command");
                    out.flush();
                    return;
            }
            
            if (!redman) {
                if (!appmon.local) {
                    out.println("NACK - State machine for " + appmon.name 
                                                        + " not configured to run on this host.");
                    out.flush();
                    return;
                }
            
                String state = appmon.getSMAState();
                trace.debug(appmon.name + ": current state is " + state);
                if (state.startsWith("IN TRANSITION") && !cmd.equals("QUERY") &&
                            !cmd.equals("RELOAD") && !cmd.equals("SHUTDOWN-F") && 
                            !cmd.equals("DISABLE-F") && !cmd.equals("RESTART-F") &&
                                                              !cmd.equals("PRIMARY")) {
                    out.println("NACK - Application " + state);
                    out.flush();
                    return;
                }
            }
            
            if (cmd.equals("START")) execStartCmd();
            else if (cmd.equals("SHUTDOWN")) execShutdownCmd(false);
            else if (cmd.equals("SHUTDOWN-F")) execShutdownCmd(true);
            else if (cmd.equals("ENABLE")) execEnableCmd();
            else if (cmd.equals("DISABLE")) execDisableCmd(false);
            else if (cmd.equals("DISABLE-F")) execDisableCmd(true);
            else if (cmd.equals("INHIBIT")) execInhibitCmd();
            else if (cmd.equals("UNINHIBIT")) execUninhibitCmd();
            else if (cmd.equals("SWITCH")) execSwitchCmd();
            else if (cmd.equals("RESTART")) execRestartCmd(false);
            else if (cmd.equals("RESTART-F")) execRestartCmd(true);
            else if (cmd.equals("QUERY")) execQueryCmd();
            else if (cmd.equals("RELOAD")) execReloadCmd();
            else if (cmd.equals("TEST")) execTestCmd();
            else if (cmd.equals("UNTEST")) execUntestCmd();
            else if (cmd.equals("ACTIVE")) execActiveCmd();
            else if (cmd.equals("STANDBY")) execStandbyCmd();
            else if (cmd.equals("PRIMARY")) execPrimaryCmd();
            else if (cmd.equals("GETPID")) execGetPidCmd();
        }
            
        private void handleOPINTConnection(String opintmsg) {            
            boolean firsttime = true;
            String msg = opintmsg, appname = null, token;
            do {
                try {
                    if (!firsttime) msg = in.readLine();
                    else firsttime = false;
                    if ((msg == null) || (msg.length() == 0)) return;
                    StringTokenizer st = new StringTokenizer(msg);
                    st.nextToken();
                    appname = st.nextToken();
                    trace.debug("opintmsg = " + msg);
             
                    if (checkApplication(msg)) {                       
                        trace.debug("OPINT connection thread: application name ok.");
                    }
                    else {
                        trace.warn("OPINT connection thread: unknown application name.");
                        out.println("Unknown application name: " + appname);
                        out.flush();
                        continue;
                    }                
                    // Get the command and process it
                    
                    String cmd = st.nextToken(); 
                    trace.debug("cmd = " + cmd);
                    if (cmd.equals("PRIMARY")) newprimaryhost = st.nextToken(); 
                    if ((cmd.equals("RESTART") || cmd.equals("RESTART-F")) 
                                                    && st.hasMoreTokens()) 
                        redmancfgfilepath = st.nextToken();
                    if (cmd.equals("GETPID")) {
                        if (redman) {
                            if (st.hasMoreTokens()) {
                                token = st.nextToken();
                                if (token.equals("BACKUP")) backup = true;
                            }
                        }
                        else procname = st.nextToken();
                    }
                    if (cmd.equals("RELOAD")) {
                        if (st.hasMoreTokens()) {
                            token = st.nextToken();
                            if (token.equals("REDUNDANT")) redundant = true;
                        }
                    }
                    
                    // Execute command
                    processOPINTCommand(cmd);
                    
                    // Turn off switches that may be on
                    if (backup) backup = false;
                    if (redundant) redundant = false;
                    if (redman) redman = false;
                    if (redmancfgfilepath != null) redmancfgfilepath = null;
                }
                catch (Exception e) {
                    trace.error("Exception: " + e.getMessage());
                }
            }
            while (true);
        }
        
        private boolean isNewConnection(String msg) {
            String token = null;
            // New primary msg = 
            // RAPP STATE <appname> <hostname> <state> <PRIMARY | SECONDARY> <INHIBITED | UNINHIBITED> NEW
            StringTokenizer st = new StringTokenizer(msg);
            for (int i=0; i < 6; i++) token = st.nextToken();
            if (token.equals("NEW")) {
                appmon.outToRemoteRedMan = null;
                return(true);
            }
            else {
                if (st.hasMoreTokens()) {
                    token = st.nextToken();
                    if (token.equals("NEW")) {
                        appmon.outToRemoteRedMan = null;
                        return(true);
                    }
                    else return(false);
                }
                else return(false);
            }
        }
        
        private void comparePrimaryHostConfig(String msg) {
            String token = null;
            // New primary msg = 
            // RAPP STATE <appname> <hostname> <state> <PRIMARY | SECONDARY> <INHIBITED | UNINHIBITED> NEW
            StringTokenizer st = new StringTokenizer(msg);
            for (int i=0; i < 6; i++) token = st.nextToken();
            if ((token.equals("PRIMARY") && appmon.designation) ||
                (token.equals("SECONDARY") && !appmon.designation)) {
                appmon.resolvePrimaryDesignation();
            }
            else {
                trace.debug(appmon.name + ": primary designation is consistent");
            }
        }
        
        private void compareFailoverEnabledConfig(String msg) {
            String token = null;
            // New primary msg = 
            // RAPP STATE <appname> <hostname> <state> <PRIMARY | SECONDARY> <INHIBITED | UNINHIBITED> NEW
            StringTokenizer st = new StringTokenizer(msg);
            for (int i=0; i < 7; i++) token = st.nextToken();
            if ((token.equals("INHIBITED") && appmon.failoverenabled) ||
                (token.equals("UNINHIBITED") && !appmon.failoverenabled)) {
                appmon.resolveFailoverEnabledConfiguration();
            }
            else {
                trace.debug(appmon.name + ": primary designation is consistent");
            }
        }
        
        private void handleRAPPConnection(String rappmsg) {
            try {                        
                if (checkApplication(rappmsg)) {
                    trace.debug(appmon.name 
                        + " - RAPP connection thread: msg passed rem app test. rappmsg = " + rappmsg);
                    processRemStateChange(rappmsg);
                }
                else {
                    trace.warn(appmon.name + ": conn thread msg \"" 
                                           + rappmsg + " did not pass rem app test");
                    return;
                }  
                
                // Compare primary configurations and take appropriate action if they differ
                comparePrimaryHostConfig(rappmsg);
                
                // Compare FAILOVER_ENABLED flag configurations and take appropriate action
                // if they differ
                compareFailoverEnabledConfig(rappmsg);
                
                while ((appmon.outToRemoteRedMan == null) && (appmon.remredmancon != null)) {
                    trace.debug("handleRAPPConnection(): there is a remredmancon thread trying to connect to the remote RedMan. Will sleep ...");
                    try {
                        sleep(3000);
                    }
                    catch (Exception e) {
                        trace.warn("handleRAPPConnection() sleep interrupted");
                    }
                }

                // Check if the application monitor is connected to the remote RedMan
                // to send state messages. If not, start thread to establish the connection.
                if (appmon.outToRemoteRedMan == null) {
                    trace.warn(appmon.name + ": no outbound connection to remote RedMan. Will connect");
                    appmon.startThreadToConnectToRemApp();
                }
                else if (isNewConnection(rappmsg)) {
                    trace.warn(appmon.name + ": new connection to remote RedMan requested. Will reconnect");
                    appmon.startThreadToConnectToRemApp();
                }
                
                // Receive messages from remote RedMan
                for (String msg = null; (msg = in.readLine()) != null;) {
                    trace.info(appmon.name + ": conn thread received msg \"" 
                                           + msg + "\", length = " + msg.length());
                    if (msg.length() == 0) break;
                    if (isRemStateChange(msg)) processRemStateChange(msg);
                    else if (isRemCommand(msg)) processRemCommand(msg);
                    else if (isCommCheck(msg)) {
                        trace.debug("Will call handleHandShake()...");
                        appmon.handleHandShake();
                    }
                }
                trace.debug(appmon.name + " - RAPP connection thread out of for loop. Will exit ...");
                //appmon.outToRemoteRedMan = null;
            }
            catch (Exception e) {
                trace.error("Exception: " + e.getMessage());
            }
        }
        
        
        public void run() {      // HandleClientThread          
            String msg = null, resp = null;
            
            try {                
                // Read first message
                trace.info("A client thread was created. Will call readLine()");
                msg = in.readLine();
                trace.info("Connection thread got out of readLine");
                if (msg == null) {
                    trace.info("Connection thread got initial null message");
                    return;
                }
                if (msg.length() == 0) {
                    trace.info("Connection thread got initial zero-length message");
                    return;
                }
                trace.info("User/Password msg = '" + msg + "'");
                resp = ConnManager.authenticate(msg);
                if (resp.equals("ACK")) {
                    trace.info("Connection Manager authenticated login - sending ACK");
                    out.println(resp);
                    out.flush();
                }
                else {
                    trace.info("Connection Manager did not authenticate login - sending '" 
                                                                    + resp + "'");
                    out.println(resp);
                    out.flush();
                    return;
                }
                
                msg = in.readLine();
                if (msg == null) {
                    trace.info("Connection thread got null message");
                    return;
                }
                if (msg.length() == 0) {
                    trace.info("Connection thread got zero-length message");
                    return;
                }                
                trace.info("Connection thread received first msg \"" + msg + "\"");
                
                // Get the type of the message
                StringTokenizer st = new StringTokenizer(msg);
                String type = st.nextToken();
                connType = type;
                
                if (type.equals("APP")) handleAPPConnection(msg);
                if (type.equals("RAPP")) handleRAPPConnection(msg); 
                if (type.equals("PRXY")) handlePRXYConnection(msg);
                if (type.equals("OPINT")) handleOPINTConnection(msg);
            }
            catch (Exception e) {
                trace.error("Exception: " + e.getMessage());
            }
        }
    }
